import json
import http.client
from selenium import webdriver
import re

#自定义请求头
custom_headers = {"Host":"segmentfault.com",
      "User-Agent":"Mozilla/5.0 (X11; Fedora; Linux x86_64; rv:55.0) Gecko/20100101 Firefox/55.0",
      "Accept":"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
      "Accept-Language":"en-US,en;q=0.5",
      "Accept-Encoding":"gzip, deflate, br",
      "Connection":"keep-alive"
      }


#去掉非标准的正则表达式
def pattern_strip(str):
    if r"\;" in str:
        return str[:str.find(r"\;")]
    else:
        return str

#解析url获取页面内容和响应头
def load_url(url):
    driver = webdriver.PhantomJS("/bin/phantomjs")
    driver.get(url)

    #获取网址的主机域名
    host = url
    host = host[host.find("//") + 2:]
    if ("/" in host):
        host = host[:host.find("/")]

    if(re.search(re.compile(r'^https'), url) != None):
        connection = http.client.HTTPSConnection(host)
    else:
        connection = http.client.HTTPConnection(host)

    connection.request("HEAD", url)
    response = connection.getresponse()
    if(response.getcode() != 200):
        return (None, None)

    web_headers = response.getheaders()
    return (driver, web_headers)


# 检查环境变量
def check_env(driver, param):
    #print(type(param))
    #print(param)

    #获取网页内的全局变量
    envs = driver.execute_script(r"return Object.getOwnPropertyNames(window)")
    if type(param) == list:
        for var1 in param:
            mpattern = re.compile(pattern_strip(var1))
            for var2 in envs:
                if re.search(mpattern, var2) != None:
                    return True
    else:

        mpattern = re.compile(pattern_strip(param))
        for var2 in envs:
            if re.search(mpattern, var2) != None:
                return True

    return False


#检查网页headers
def check_headers(headers, param):
    #将列表形式的headers转为字典
    headers_dict = dict(headers)
    for var1 in param:
        if var1 in headers_dict:
            mpattern = re.compile(pattern_strip(param[var1]))
            if re.search(mpattern, headers_dict[var1]):
                return True

    return False

#检查网页的html
def check_html(driver, param):
    web_sources = driver.page_source

    if type(param) == list:
        for var1 in param:
            mpattern = re.compile(pattern_strip(var1))
            if re.search(mpattern, web_sources):
                return True
    else:
        mpattern = re.compile(pattern_strip(param))
        if re.search(mpattern, web_sources):
            return True

    return False


#检查URL
def check_url(url, param):
    #print(type(url))
    #print(type(param))
    mpattern = re.compile(pattern_strip(param))
    if re.search(mpattern, url):
        return True
    else:
        return False

#检查网页的meta信息
def check_meta(driver, param):
    metas = driver.find_elements_by_tag_name("meta")
    for var1  in param:
        for var2 in metas:
            if var2.get_attribute("name") == var1 and re.search(re.compile(pattern_strip(param[var1])), var2.get_attribute("content")) :
                return True

    return False

#检查网页的js标签 运行需要花费大量时间
def check_script(driver, param):
    scripts = driver.find_elements_by_tag_name("script")

    if type(param) == list:
        #print("list")
        for var1 in param:
            pattren = re.compile(pattern_strip(var1))
            for var2 in scripts:
                #print(count)
                if var2.get_attribute("src") != None and re.search(pattren, var2.get_attribute("src")):
                    return True
    else:
        #print("str")
        pattren = re.compile(pattern_strip(param))
        for var2 in scripts:
            if var2.get_attribute("src") != None and re.search(pattren, var2.get_attribute("src")):
                return True
        #print("end")

    return False


#检查网页的依赖信息 递归实现
def check_implies(result_dict, apps_json, app = None):
    if app == None:
        #dict_keys = result_dict.keys()
        for var1 in list(result_dict):
            if "implies" in apps_json["apps"][var1]:

                if type(apps_json["apps"][var1]["implies"]) == list:
                    for var2 in apps_json["apps"][var1]["implies"]:
                        implies_str = pattern_strip(var2)
                        if implies_str not in result_dict:
                            result_dict[implies_str] = {}
                        check_implies(result_dict, apps_json, implies_str)
                else:

                    implies_str = pattern_strip(apps_json["apps"][var1]["implies"])
                    if implies_str not in result_dict:
                        result_dict[implies_str] = {}
                    check_implies(result_dict, apps_json, implies_str)

    else:
        if "implies" in apps_json["apps"][app]:

            if type(apps_json["apps"][app]["implies"]) == list:
                for var2 in apps_json["apps"][app]["implies"]:
                    implies_str = pattern_strip(var2)
                    if implies_str not in result_dict:
                        result_dict[implies_str] = {}
                    check_implies(result_dict, apps_json, implies_str)

            else:
                implies_str = pattern_strip(apps_json["apps"][app]["implies"])
                if implies_str not in result_dict:
                    result_dict[implies_str] = {}
                check_implies(result_dict, apps_json, implies_str)

#web_driver, web_headers = load_url("https://segmentfault.com/a/1190000002407912")

#print(web_driver.page_source + "\n")



#print(web_driver.execute_script(r"return Object.getOwnPropertyNames(window)"))

#print(check_env(web_driver, ['SVGTextPathElement', 'encodeURIComponent']))
#print(dict(web_headers))


#对链接网址匹配指纹信息 由于性能原因，未匹配js
def web_finger_check(url):
    apps_json_fp = open("apps.json", "r")
    apps_json = json.load(apps_json_fp)
    apps_json_fp.close()

    json_len = len(apps_json["apps"])
    current_len = 0

    web_driver, web_headers = load_url(url)

    print(web_headers)


    result_dict = dict()
    apps_dict = dict()

    for var1 in apps_json["apps"]:

        sign = 0

        #print("env")
        if "env" in apps_json["apps"][var1]:
            #print(var1)
            if check_env(web_driver, apps_json["apps"][var1]["env"]) == True:
                apps_dict["env"] = True
                sign = 1
            else:
                apps_dict["env"] = False

        #print("headers")
        if "headers" in apps_json["apps"][var1]:
            if check_headers(web_headers , apps_json["apps"][var1]["headers"]) == True:
                apps_dict["headers"] = True
                sign = 1
            else:
                apps_dict["headers"] = False

        #print("html")
        if "html" in apps_json["apps"][var1]:
            if check_html(web_driver, apps_json["apps"][var1]["html"]) == True:
                apps_dict["html"] = True
                sign = 1
            else:
                apps_dict["html"] = False

        #print("url")
        if "url" in apps_json["apps"][var1]:
            if check_url(url, apps_json["apps"][var1]["url"]) == True:
                apps_dict["url"] = True
                sign = 1
            else:
                apps_dict["url"] = False

        #print("meta")
        if "meta" in apps_json["apps"][var1]:
            if check_meta(web_driver, apps_json["apps"][var1]["meta"]) == True:
                apps_dict["meta"] = True
                sign = 1
            else:
                apps_dict["meta"] = False


        '''
        #print("script")
        if "script" in apps_json["apps"][var1]:
            if check_script(web_driver, apps_json["apps"][var1]["script"]) == True:
                apps_dict["script"] = True
                sign = 1
            else:
                apps_dict["script"] = False
        #print("end")
        '''

        if sign == 1:
            result_dict[var1] = apps_dict

        apps_dict = dict()

        current_len += 1
        print(str(current_len) + "/" + str(json_len))

    print("\n")

    check_implies(result_dict, apps_json)


    return result_dict

#将匹配信息简单处理
def result_process(result_dict):
    apps_json_fp = open("apps.json", "r")
    apps_json = json.load(apps_json_fp)
    apps_json_fp.close()

    result_list = list(result_dict)

    process_result = dict()

    for var1 in result_list:
        for var2 in apps_json["apps"][var1]["cats"]:
            if apps_json["categories"][var2]["name"] not in process_result:
                process_result[apps_json["categories"][var2]["name"]] = []
            process_result[apps_json["categories"][var2]["name"]].append(var1)

    return process_result



result = web_finger_check("http://www.php.cn/php/php-tutorial.html")
print(result)
print(result_process(result))


'''
meta = web_driver.find_elements_by_tag_name("meta")

print(len(meta))

for var in meta:
    print(len(var.get_attribute("name")))
'''


#web_driver, web_headers = load_url("http://www.runoob.com/js/js-regexp.html")
#print(type(web_driver.execute_script(r"return Object.getOwnPropertyNames(window)")))

#headers = dict(web_headers)
#pattern_str = dict(json.load(open("apps.json")))["apps"]["Nginx"]["headers"]["Server"]

#scripts = web_driver.find_elements_by_tag_name("script")

#pattern = re.compile("googlesyndication\.com/")

#for var in scripts:
 #   pass
    ##print(var.get_attribute("src"))
    ##print(re.match(pattern, var.get_attribute("src")))

#print(headers["Server"])
#print(pattern_strip(pattern_str))

#print(re.search(pattern, "https://pagead2.googlesyndication.com/pub-config/r20160913/ca-pub-5751451760833794.js"))

#pattern = re.compile(r"nginx(?:/([\d.]+))?")
#print(pattern.match(headers["Server"]))